pacman::p_load(dplyr, readr, sf, tidyverse, tmap, sfdep, ggplot2, plotly, spdep, rjson, osmextract, geojsonR, rvest, httr, jsonlite)1 Overview
1.1 Aims
To evaluate which data source is most complete and suitable
OSM
data.gov.sg
To prepare the data sources for machine learning
- Combination of data sources, fields and manual intervention
2 Getting Started
2.1 Setting Up
Packages required to be loaded for
2.2 Data Sources
| Dataset Name | Source | Methodology |
|---|---|---|
| HDB Existing Buildings | Data.gov.sg | Download |
| HDB Property Information (4 Aug 2024) | Data.gov.sg | Download |
| URA Master Plan 2019 Land Use (3 Jul 2023) | Data.gov.sg | Download |
| Humanitarian OSM Team Buildings Dataset (1 July 2024) | HOTOSM | Download |
3 Key Functions
3.1 Onemap
get_postal <- function(addr_lst){
# Create a data frame to store all retrieved coordinates
new_postal <- data.frame()
i = 0
for (addr in addr_lst){
i = i + 1
if (i %% 100 == 0){
print(c(addr, as.character(i)))
}
reply <- GET('https://www.onemap.gov.sg/api/common/elastic/search?',
query = list(searchVal = addr,
returnGeom = 'N',
getAddrDetails = 'Y'))
output <- fromJSON(rawToChar(reply$content))
found <- output$found
res <- output$results
# Create a new data frame for each address
new_row <- data.frame()
# If single result, append
if (found >= 1){
res_1 <- head(res, n = 1)
postal <- res_1$POSTAL
new_row <- data.frame(address = addr, postal = postal)
}
else {
new_row <- data.frame(address = addr, postal = NA)
}
# Add the row
new_postal <- rbind(new_postal, new_row)
remove(new_row)
}
return(new_postal)
}token = ""
get_postal_rev_geocode <- function(addr_lst){
# Create a data frame to store all retrieved coordinates
new_postal <- data.frame()
for (i in 1:nrow(addr_lst)){
if (i == 100){
print(c(addr_lst[i,"X"][[1]], addr_lst[i,"Y"][[1]], as.character(i)))
}
query1 <- paste(addr_lst[i,"X"][[1]], addr_lst[i,"Y"][[1]], sep = ",")
reply <- GET('https://www.onemap.gov.sg/api/public/revgeocodexy?',
query = list(location = query1,
buffer = '50',
addressType = 'HDB'),
add_headers("Content-Type"="application/json",
Accept="application/+json",
"Authorization"= paste("Bearer", token)))
output <- fromJSON(rawToChar(reply$content))
res <- output$GeocodeInfo
found <- length(res)
# Create a new data frame for each address
new_row <- data.frame()
# If single result, append
if (found >= 1){
res_1 <- head(res, n = 1)
if ((res_1$POSTALCODE) == "NIL"){
postal <- NA
}
else{
postal <- res_1$POSTALCODE
}
new_row <- data.frame(X = addr_lst[i,"X"][[1]], Y = addr_lst[i, "Y"][[1]], postal = postal)
}
else {
new_row <- data.frame(X = addr_lst[i,"X"][[1]], Y = addr_lst[i, "Y"][[1]], postal = NA)
}
# Add the row
new_postal <- rbind(new_postal, new_row)
}
return(new_postal)
}4 Data Preparation
4.1 Loading Data
Loading HDB Property Information (4 Aug 2024)
RAW_HDB_PROPERTY <- read.csv("data/fyp_preprocessing/HDBPropertyInformation.csv")Loading HDB Existing Buildings KML
raw_hdb_bldgs = st_read("data/fyp_preprocessing/HDBExistingBuilding.geojson")Reading layer `HDBExistingBuilding' from data source
`C:\Users\renji\OneDrive - Singapore Management University\0_git-projects\urbancoalesce\explore\data\fyp_preprocessing\HDBExistingBuilding.geojson'
using driver `GeoJSON'
Simple feature collection with 12847 features and 2 fields
Geometry type: MULTIPOLYGON
Dimension: XY, XYZ
Bounding box: xmin: 103.6848 ymin: 1.270005 xmax: 103.989 ymax: 1.457245
z_range: zmin: 0 zmax: 0
Geodetic CRS: WGS 84attributes <- lapply(X = 1:nrow(raw_hdb_bldgs),
FUN = function(x) {
raw_hdb_bldgs %>%
slice(x) %>%
pull(Description) %>%
read_html() %>%
html_node("table") %>%
html_table(header = TRUE, trim = TRUE, dec = ".", fill = TRUE) %>%
as_tibble(.name_repair = ~ make.names(c("Attribute", "Value"))) %>%
pivot_wider(names_from = Attribute, values_from = Value)
})
hdb_bldgs <-
raw_hdb_bldgs %>%
bind_cols(bind_rows(attributes)) %>%
select(-Description)
hdb_bldgs <- st_zm(hdb_bldgs)
hdb_bldgs <- st_transform(hdb_bldgs, crs = 3414)Loading Humanitarian OSM Buildings Dataset (1 July 2024)
osm_bldgs <- st_read("data/fyp_preprocessing/QUICKOSM_BLDGS_20240818_QGIS/osmbldgs_20240818.gpkg")Reading layer `building_singapore' from data source
`C:\Users\renji\OneDrive - Singapore Management University\0_git-projects\urbancoalesce\explore\data\fyp_preprocessing\QUICKOSM_BLDGS_20240818_QGIS\osmbldgs_20240818.gpkg'
using driver `GPKG'
Simple feature collection with 121717 features and 477 fields
Geometry type: MULTIPOLYGON
Dimension: XY
Bounding box: xmin: 103.6059 ymin: 1.159946 xmax: 104.4063 ymax: 1.470583
Geodetic CRS: WGS 84osm_bldgs <- osm_bldgs %>%
st_transform(crs = 3414)TO-DO: EXTRACT HOTOSM DATA BASED ON MP19 LAND USE POLYGON = RESIDENTIAL (CAN SELECT HDB ONLY?)
tmap_options(check.and.fix = TRUE)
tmap_mode("plot")
tm_shape(osm_bldgs) +
tm_fill(col = "building")
unique(osm_bldgs[['building']]) [1] "yes" "garage"
[3] "commercial" "school"
[5] "industrial" "warehouse"
[7] "residential" "retail"
[9] "hotel" "church"
[11] "college" "hospital"
[13] "office" "civic"
[15] "public" "roof"
[17] "apartments" "education"
[19] "parking" "kindergarten"
[21] "university" "dormitory"
[23] "stable" "house"
[25] "detached" "yes;industrial"
[27] "construction" "government"
[29] "temple" "bridge"
[31] "public;roof" "commercial;yes"
[33] "train_station" "transportation"
[35] "service" "religious"
[37] NA "greenhouse"
[39] "hut" "mosque"
[41] "hangar" "sports_centre"
[43] "grandstand" "waste_transfer_station"
[45] "manufacture" "boathouse"
[47] "clinic" "terrace"
[49] "semidetached_house" "chapel"
[51] "toilets" "ruins"
[53] "storage_tank" "fire_station"
[55] "synagogue" "swimming_pool_changing_room"
[57] "stadium" "mix_used"
[59] "shop" "guardhouse"
[61] "shed" "gazebo"
[63] "shrine" "CET_Campus_East"
[65] "wayside_shrine" "EiS_Residences"
[67] "medical" "IMM"
[69] "farm_auxiliary" "bungalow"
[71] "sports_hall" "hall"
[73] "carport" "monastery"
[75] "utility" "multi-purpose_stage"
[77] "no" "security"
[79] "pavilion" "gateway"
[81] "seasonal" "pagoda"
[83] "switchroom" "garages"
[85] "yes;retail" "Security_Post"
[87] "supermarket" "jtc_nanospace"
[89] "gatehouse" "multi-purpose_hall"
[91] "tent" "shelter"
[93] "tourism" "2"
[95] "kiosk" "bunker"
[97] "healthcare" Key focus
residential
mix_used
yes
2?
apartments
house
temp <- osm_bldgs %>% filter(building == "yes")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")Drop mix_used
temp <- osm_bldgs %>% filter(building == "mix_used")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")temp <- osm_bldgs %>% filter(building == "yes")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")Drop 2 - facility in boon lay
temp <- osm_bldgs %>% filter(building == "2")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")Keep - how to manage the HDB and condo and split them out?
temp <- osm_bldgs %>% filter(building == "apartments")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")temp <- osm_bldgs %>% filter(building == "house")
tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(temp) +
tm_fill(col = "building")EDA Summary
Retain following fields:
residential - some overlap with condo
yes - overlap with many other non residential
apartments - overlap with condo
proc_osm_bldgs <- osm_bldgs %>% filter(building %in% c("residential", "yes", "apartments"))
proc_osm_bldgs <- proc_osm_bldgs %>% filter(is.na(addr.street) == FALSE)
proc_osm_bldgs <- proc_osm_bldgs %>%
mutate(proc_osm_bldgs, address = ifelse(is.na(addr.housenumber) == FALSE, paste(addr.housenumber, addr.street),addr.street))Moving forward: combine with HDB data, check and drop remainder with no match with HDB dataset
addr_lst <- sort(unique(proc_osm_bldgs$address))
postal <- get_postal(addr_lst)
write_rds(postal, "data/fyp_preprocessing/osm_bldgs_postal.rds")OSM_BLDGS_POSTAL <- read_rds("data/fyp_preprocessing/osm_bldgs_postal.rds")
temp_osm <- left_join(proc_osm_bldgs, OSM_BLDGS_POSTAL, by=c("address" = "address"))temp_view <- temp_osm %>% filter(postal == "NIL")
unique(temp_view$address) [1] "Outram Road" "Seah Street"
[3] "Kim Seng Road" "Jalan Haji Alias"
[5] "Coronation Road West" "Kaki Bukit Avenue 1"
[7] "Sin Ming Drive" "Tampines North Drive 2"
[9] "Tampines Road" "710A Ang Mo Kio Central 1"
[11] "714 Ang Mo Kio Central 1" "Changi Village Road"
[13] "419 Bedok North Avenue 1" "420 Bedok North Avenue 1"
[15] "Simei Street 6" "Heng Mui Keng Terrace"
[17] "Jalan Pekan Ubin" "Jalan Batu Ubin"
[19] "Woodlands Square" "Simei Street 4"
[21] "Ang Mo Kio Industrial Park 2A" "980 Buangkok Green"
[23] "Riverside Road" "Joo Koon Circle"
[25] "10 Tuas Avenue 10" "15 Tuas Street"
[27] "13 Tuas Avenue 13" "Tuas Avenue 11"
[29] "11 Tuas Avenue 11" "Gul Street 2"
[31] "Yuk Tong Avenue" "Defu Lane 4"
[33] "Defu Lane 3" "Joo Koon Way"
[35] "Joo Koon Road" "Pandan Road"
[37] "Bukit Batok West Avenue 5" "Kallang Way"
[39] "Fusionopolis Walk" "Loyang Avenue"
[41] "Senoko Drive" "Telok Ayer Street"
[43] "Farrer Road" "Purvis Street"
[45] "2 Seragoon North Avenue 2" "Coral Island"
[47] "Ocean Drive" "Sandy Island"
[49] "163 Bukit Batok West Avenue 8" "162 Bukit Batok West Avenue 8"
[51] "155 Bukit Batok West Avenue 8" "Bukit Batok East Avenue 6"
[53] "12 Artillery Avenue" "Bukit Batok Street 31"
[55] "Margaret Drive" "91 Hougang Avenue 3"
[57] "85 Hougang Avenue 3" "Defu Lane 1"
[59] "Gambas Avenue" "Senoko Crescent"
[61] "Maxwell Road" "Macalister Road"
[63] "114 Bukit Purmei Avenue" "Central Expressway"
[65] "113 Bukit Purmei Avenue" "110 Bukit Purmei Avenue"
[67] "115 Bukit Purmei Avenue" "112 Bukit Purmei Avenue"
[69] "111 Bukit Purmei Avenue" "Delta Road"
[71] "Peng Nguan Street" "103 Henderson Road"
[73] "44A Beo Crescent" "Ganges Avenue"
[75] "Clarence Lane" "Penang Road"
[77] "Moulmein Road" "Bencoolen Street"
[79] "215 Airport Road" "Queensway"
[81] "201 Commonwealth Avenue West" "Yishun Ave 6"
[83] "Cavenagh Road" "Defu Avenue 1"
[85] "50 Clive Road" "Hoot Kiam Road"
[87] "Paterson Road" "Wilkie Road"
[89] "Joo Chiat Road" "Jalan Sultan"
[91] "18 Baboo Lane" "Kelantan Road"
[93] "Mount Faber Road" "Fernvale Road"
[95] "Suffolk Road" "33 Chai Chee Road"
[97] "34 Chai Chee Road" "35 Chai Chee Road"
[99] "36 Chai Chee Road" "37 Chai Chee Road"
[101] "40 Chai Chee Road" "305D Punggol Road"
[103] "Chapel Road" "Draycott Drive"
[105] "215 Choa Chu Kang Central" "216 Choa Chu Kang Central"
[107] "Choa Chu Kang North 5" "Ubi Road 3"
[109] "Race Course Lane" "Jansen Road"
[111] "Kovan Road" "Highland Road"
[113] "Tuas View Crescent" "Cove Drive"
[115] "Grange Road" "Emerald Hill Road"
[117] "Mandalay Road" "Bedok Reservoir Road"
[119] "Hougang Street 11" "Joo Chiat Place"
[121] "Braddell Road" "Mackerrow Road"
[123] "3188 Geylang Bahru" "192C Punggol Road"
[125] "Oxford Road" "Pandan Gardens"
[127] "Changi Road" "Everitt Road"
[129] "Sembawang Road" "Bedok Road"
[131] "Jalan Chempaka Kuning" "Koon Seng Road"
[133] "Bukit Timah Road" "Changi South Avenue 1"
[135] "Siglap Road" "Ceylon Road"
[137] "Turnhouse Road" "Netheravon Road"
[139] "Lorong Marzuki" "Alexandra Road"
[141] "85 Bedok North Avenue 4" "Toh Tuck Link"
[143] "Saint Martin's Drive" "Chander Road"
[145] "13 Jalan Sentosa" "32 Chai Chee Road"
[147] "73N Jalan Senang" "91L Jalan Senang"
[149] "Pioneer Walk" "Pioneer Turn"
[151] "Guillemard Lane" "Aljunied Road"
[153] "Lorong 24 Geylang" "28 Lorong 28 Geylang"
[155] "Lorong 28 Geylang" "Lorong 30 Geylang"
[157] "34 Lorong 34 Geylang" "Lorong 34 Geylang"
[159] "Serangoon Road" "89 Bedok North Avenue 4"
[161] "87 Bedok North Avenue 4" "88 Bedok North Avenue 4"
[163] "Kampong Bahru Road" "440C Clementi Close"
[165] "293D Bukit Batok East Avenue 6" "460A Bukit Batok West Avenue 8"
[167] "460B Bukit Batok West Avenue 8" "463B Bukit Batok West Avenue 8"
[169] "Boscombe Road" "Parkstone Road"
[171] "224-J Jalan Endut Senin" "Jalan Endut Senin"
[173] "84 Bedok North Avenue 4" "86 Bedok North Avenue 4"
[175] "90 Bedok North Avenue 4" "91 Bedok North Avenue 4"
[177] "Chai Chee Road" "Bendemeer Road"
[179] "Cavan Road" "Hamilton Road"
[181] "Syed Alwi Road" "1003 Bukit Teresa Road"
[183] "Jalan Bukit Ho Swee" "Hawk Arena"
[185] "439 Clementi Close" "Segar Road"
[187] "Woodgrove View" "Bloxhome Drive"
[189] "Duchess Road" "Serangoon Garden Way"
[191] "Palm Grove Avenue" "Jalan Ampang"
[193] "Glasgow Road" "Sandilands Road"
[195] "Sin Ming Road" "Charlton Lane"
[197] "Lorong Kilat" "Joo Chiat Avenue"
[199] "Ulu Pandan Park Connector" "Chiap Guan Avenue"
[201] "Joo Chiat Walk" "107"
[203] "Puay Hee Avenue" "Choa Chu Kang Road"
[205] "Galistan Avenue" "19 Harding Road"
[207] "Parry Avenue" "Swiss Club Avenue"
[209] "Dunearn Road" "Poh Huat Road"
[211] "20 Peirce Road" "Faber Drive"
[213] "Lorong 6 Realty Park" "Lichi Avenue"
[215] "Woodgrove Drive" "Jalan Usaha"
[217] "Sungei Kadut Avenue" "Upper Paya Lebar Road"
[219] "Paya Lebar Crescent" "Lorong Ong Lye"
[221] "East Coast Avenue" "Tai Hwan Heights"
[223] "Lothian Terrace" "16 The Oval Seletar Aerospace Park"
[225] "5A Bays Water Road" "Bodmin Drive"
[227] "Carisbrooke Grove" "Burghley Drive"
[229] "Chuan Walk" "Figaro Street"
[231] "Palm Drive" "Chaun Drive"
[233] "Elite Terrace" "Canberra Road"
[235] "Tuas Road" "Tagore Drive"
[237] "Tagore Industrial Avenue" "Tagore Lane"
[239] "Mulberry Avenue" "Pasir Ris View"
[241] "Riverina Crescent" "Jalan Pelatina"
[243] "Cactus Road" "Siglap Bank"
[245] "Burnfoot Terrace" "Thomson Green"
[247] "Harvey Avenue" "Sommerville Road"
[249] "Jalan Ria" "Jalan Riang"
[251] "Jalan Ishak" "Lorong Marican"
[253] "Lorong Biawak" "Flora Road"
[255] "Kaki Bukit Road 3" "Simei Rise"
[257] "Beatty Road" "Lorong 19 Geylang"
[259] "11 Lorong 11 Geylang" "Lorong 14 Geylang"
[261] "Cardiff Grove" "Conway Circle"
[263] "Jalan Limbok" "Lentor Street"
[265] "Lentor Link" "Lentor Green"
[267] "Bedok South Avenue 1" "Lim Ah Pin Road"
[269] "Tampines Street 83" "18 Spottiswoode"
[271] "Loyang Way" "Changi Business Park Vista"
[273] "Bowmont Gardens" "Lorong 22 Geylang"
[275] "Paya Lebar Road" "Lynwood Grove"
[277] "Simon Road" "Loewen Road"
[279] "Tuas South Avenue 3" "Yishun Avenue 4"
[281] "Coronation Road" "Imbiah Road"
[283] "China Street" "Sixth Lok Yang Road"
[285] "Woodlands Road" "River Valley Road"
[287] "Choa Chu Kang Drive" "Jurong East Street 32"
[289] "Ang Mo Kio Central 3" "Bishan Street 23"
[291] "Faber Walk" "Yishun Street 23"
[293] "Benoi Sector" "Benoi Road"
[295] "Pheng Geck Avenue" "Tong Watt Road"
[297] "Sea Avenue" "Ang Mo Kio Avenue 1"
[299] "West Coast Way" "28A West Jalan Lempeng"
[301] "28A East Jalan Lempeng" "13 Clementi Crescent"
[303] "1007 Bukit Teresa Road" "1005 Bukit Teresa Road"
[305] "York Place" "Bishan Street 15"
[307] "Bulim Avenue" "Khiang Guan Avenue"
[309] "Straits View" "13 Leo Drive"
[311] "Kinta Road" "468I Upper Changi Road"
[313] "Cove Grove" "Riviera Drive"
[315] "Bedok Rise" "Tampines Street 61"
[317] "Keppel Bay View" "20 Hindhede Place"
[319] "Tampines Street 64" "Canberra Drive"
[321] "Dakota Crescent" "Clementi West Street 2"
[323] "56E Marne Road" "Mount Vernon Road"
[325] "Marsiling Road" "Chong Kuo Road"
[327] "Pemimpin Place" "Pemimpin Terrace"
[329] "15 Lorong 15 Geylang" "Poh Huat Drive"
[331] "Sunrise Terrace" "Jalan Isnin"
[333] "Jalan Minggu" temp_view <- temp_osm %>% filter(postal == "NIL")
temp_view <- temp_view %>% filter(as.numeric(substr(address, 1, 2)) >= 0)
unique(temp_view$address) [1] "710A Ang Mo Kio Central 1" "714 Ang Mo Kio Central 1"
[3] "419 Bedok North Avenue 1" "420 Bedok North Avenue 1"
[5] "980 Buangkok Green" "10 Tuas Avenue 10"
[7] "15 Tuas Street" "13 Tuas Avenue 13"
[9] "11 Tuas Avenue 11" "2 Seragoon North Avenue 2"
[11] "163 Bukit Batok West Avenue 8" "162 Bukit Batok West Avenue 8"
[13] "155 Bukit Batok West Avenue 8" "12 Artillery Avenue"
[15] "91 Hougang Avenue 3" "85 Hougang Avenue 3"
[17] "114 Bukit Purmei Avenue" "113 Bukit Purmei Avenue"
[19] "110 Bukit Purmei Avenue" "115 Bukit Purmei Avenue"
[21] "112 Bukit Purmei Avenue" "111 Bukit Purmei Avenue"
[23] "103 Henderson Road" "44A Beo Crescent"
[25] "215 Airport Road" "201 Commonwealth Avenue West"
[27] "50 Clive Road" "18 Baboo Lane"
[29] "33 Chai Chee Road" "34 Chai Chee Road"
[31] "35 Chai Chee Road" "36 Chai Chee Road"
[33] "37 Chai Chee Road" "40 Chai Chee Road"
[35] "305D Punggol Road" "215 Choa Chu Kang Central"
[37] "216 Choa Chu Kang Central" "3188 Geylang Bahru"
[39] "192C Punggol Road" "85 Bedok North Avenue 4"
[41] "13 Jalan Sentosa" "32 Chai Chee Road"
[43] "73N Jalan Senang" "91L Jalan Senang"
[45] "28 Lorong 28 Geylang" "34 Lorong 34 Geylang"
[47] "89 Bedok North Avenue 4" "87 Bedok North Avenue 4"
[49] "88 Bedok North Avenue 4" "440C Clementi Close"
[51] "293D Bukit Batok East Avenue 6" "460A Bukit Batok West Avenue 8"
[53] "460B Bukit Batok West Avenue 8" "463B Bukit Batok West Avenue 8"
[55] "224-J Jalan Endut Senin" "84 Bedok North Avenue 4"
[57] "86 Bedok North Avenue 4" "90 Bedok North Avenue 4"
[59] "91 Bedok North Avenue 4" "1003 Bukit Teresa Road"
[61] "439 Clementi Close" "107"
[63] "19 Harding Road" "20 Peirce Road"
[65] "16 The Oval Seletar Aerospace Park" "11 Lorong 11 Geylang"
[67] "18 Spottiswoode" "28A West Jalan Lempeng"
[69] "28A East Jalan Lempeng" "13 Clementi Crescent"
[71] "1007 Bukit Teresa Road" "1005 Bukit Teresa Road"
[73] "13 Leo Drive" "468I Upper Changi Road"
[75] "20 Hindhede Place" "56E Marne Road"
[77] "15 Lorong 15 Geylang" Total 77
temp_osm[temp_osm$address == "88 Bedok North Avenue 4", "address"] = "88 BEDOK NORTH STREET 4"
temp_osm[temp_osm$address == "201 Commonwealth Avenue West", "address"] = "201 CLEMENTI AVENUE 6"
temp_osm[temp_osm$address == "112 Bukit Purmei Avenue", "address"] = "112 Bukit Purmei Rd"
temp_osm[temp_osm$address == "114 Bukit Purmei Avenue", "address"] = "114 Bukit Purmei Rd"
temp_osm[temp_osm$address == "35 Chai Chee Road", "address"] = "35 Chai Chee Ave"
temp_osm[temp_osm$address == "40 Chai Chee Road", "address"] = "40 Chai Chee Ave"
temp_osm[temp_osm$address == "87 Bedok North Avenue 4", "address"] = "87 Bedok North Street 4"
temp_osm[temp_osm$address == "34 Chai Chee Road", "address"] = "34 Chai Chee Ave"
temp_osm[temp_osm$address == "36 Chai Chee Road", "address"] = "36 Chai Chee Ave"
temp_osm[temp_osm$address == "163 Bukit Batok West Avenue 8", "address"] = "163 Bukit Batok Street 11"
temp_osm[temp_osm$address == "115 Bukit Purmei Avenue", "address"] = "115 Bukit Purmei Rd"
temp_osm[temp_osm$address == "163 Bukit Batok West Avenue 8", "address"] = "163 Bukit Batok Street 11"
temp_osm[temp_osm$address == "44A Beo Crescent", "address"] = "44A Havelock Rd"
temp_osm[temp_osm$address == "714 Ang Mo Kio Central 1", "address"] = "714 Ang Mo Kio Avenue 6"
temp_osm[temp_osm$address == "163 Bukit Batok West Avenue 8", "address"] = "163 Bukit Batok Street 11"
temp_osm[temp_osm$address == "113 Bukit Purmei Avenue", "address"] = "113 Bukit Purmei Rd"
temp_osm[temp_osm$address == "162 Bukit Batok West Avenue 8", "address"] = "162 Bukit Batok Street 11"
temp_osm[temp_osm$address == "113 Bukit Purmei Avenue", "address"] = "113 Bukit Purmei Rd"
temp_osm[temp_osm$address == "103 Henderson Road", "address"] = "103 Henderson Crescent"
temp_osm[temp_osm$address == "91 Bedok North Avenue 4", "address"] = "91 Bedok North Street 4"
temp_osm[temp_osm$address == "460B Bukit Batok West Avenue 8", "address"] = "460B Bukit Batok West Avenue 9"
temp_osm[temp_osm$address == "90 Bedok North Avenue 4", "address"] = "90 Bedok North Street 4"
temp_osm[temp_osm$address == "110 Bukit Purmei Avenue", "address"] = "110 Bukit Purmei Rd"
temp_osm[temp_osm$address == "420 Bedok North Avenue 1", "address"] = "420 Bedok North Street 1"
temp_osm[temp_osm$address == "419 Bedok North Avenue 1", "address"] = "419 Bedok North Street 1"
temp_osm[temp_osm$address == "37 Chai Chee Road", "address"] = "37 Chai Chee Ave"
temp_osm[temp_osm$address == "33 Chai Chee Road", "address"] = "33 Chai Chee Ave"
temp_osm[temp_osm$address == "32 Chai Chee Road", "address"] = "32 Chai Chee Ave"
temp_osm[temp_osm$address == "89 Bedok North Avenue 4", "address"] = "89 Bedok North Street 4"
temp_osm[temp_osm$address == "86 Bedok North Avenue 4", "address"] = "86 Bedok North Street 4"
temp_osm[temp_osm$address == "293D Bukit Batok East Avenue 6", "address"] = "293D Bukit Batok Street 21"
temp_osm[temp_osm$address == "155 Bukit Batok West Avenue 8", "address"] = "155 Bukit Batok Street 11"
temp_osm[temp_osm$address == "460A Bukit Batok West Avenue 8", "address"] = "460A Bukit Batok West Avenue 9"
temp_osm[temp_osm$address == "463B Bukit Batok West Avenue 8", "address"] = "463B Bukit Batok West Avenue 9"
temp_osm[temp_osm$address == "84 Bedok North Avenue 4", "address"] = "84 Bedok North Street 4"
temp_osm[temp_osm$address == "85 Bedok North Avenue 4", "address"] = "85 Bedok North Street 4"
temp_osm[temp_osm$address == "111 Bukit Purmei Avenue", "address"] = "111 Bukit Purmei Rd"
temp_osm[temp_osm$address == "440C Clementi Close", "address"] = "440C Clementi Ave 3"
temp_osm[temp_osm$address == "439 Clementi Close", "address"] = "439 Clementi Ave 3"
temp_osm[temp_osm$address == "710A Ang Mo Kio Central 1", "address"] = "710A Ang Mo Kio Avenue 8"
temp_osm[temp_osm$address == "192C Punggol Road", "address"] = "192C Punggol Central"
temp_osm[temp_osm$address == "305D Punggol Road", "address"] = "305D Punggol Dr"
temp_osm[temp_osm$address == "980 Buangkok Green", "address"] = "980 Buangkok Cres"Update 42 useful ones - HDB - 2 postal below
Total left = 30
Rerun get address for those updated (POSTAL NIL - 42)
upd_proc_osm <- temp_osm %>% filter(postal == "NIL")
addr_lst <- sort(unique(upd_proc_osm$address))
postal <- get_postal(addr_lst)
write_rds(postal, "data/fyp_preprocessing/osm_append_bldgs_postal.rds")OSM_BLDGS_POSTAL <- read_rds("data/fyp_preprocessing/osm_bldgs_postal.rds")
OSM_BLDGS_APPEND_POSTAL <- read_rds("data/fyp_preprocessing/osm_append_bldgs_postal.rds")
OSM_COMBINE_POSTAL <- rbind(OSM_BLDGS_POSTAL, OSM_BLDGS_APPEND_POSTAL)
temp_osm$postal <- NULL
temp_osm <- left_join(temp_osm, OSM_COMBINE_POSTAL, by=c("address" = "address"))temp_osm[temp_osm$address == "215 Choa Chu Kang Central", "postal"] = "680215"
temp_osm[temp_osm$address == "216 Choa Chu Kang Central", "postal"] = "680216"temp_view <- temp_osm %>% filter(postal == "NIL")
temp_view <- temp_view %>% filter(as.numeric(substr(address, 1, 2)) >= 0)
unique(temp_view$address) [1] "10 Tuas Avenue 10" "15 Tuas Street"
[3] "13 Tuas Avenue 13" "11 Tuas Avenue 11"
[5] "2 Seragoon North Avenue 2" "12 Artillery Avenue"
[7] "91 Hougang Avenue 3" "85 Hougang Avenue 3"
[9] "215 Airport Road" "50 Clive Road"
[11] "18 Baboo Lane" "3188 Geylang Bahru"
[13] "13 Jalan Sentosa" "73N Jalan Senang"
[15] "91L Jalan Senang" "28 Lorong 28 Geylang"
[17] "34 Lorong 34 Geylang" "224-J Jalan Endut Senin"
[19] "1003 Bukit Teresa Road" "107"
[21] "19 Harding Road" "20 Peirce Road"
[23] "16 The Oval Seletar Aerospace Park" "11 Lorong 11 Geylang"
[25] "18 Spottiswoode" "28A West Jalan Lempeng"
[27] "28A East Jalan Lempeng" "13 Clementi Crescent"
[29] "1007 Bukit Teresa Road" "1005 Bukit Teresa Road"
[31] "13 Leo Drive" "468I Upper Changi Road"
[33] "20 Hindhede Place" "56E Marne Road"
[35] "15 Lorong 15 Geylang" 35 remaining - correct
4.2 Preparing HDB Property Dataset
HDB Property dataset lacks postal code which allows us to combine the attributes with the geospatial datasets. Onemap search API is used to search for the postal code with the dataset’s address line.
Adding Postal Codes to HDB Property Datasets - prepare to combine with HDB Buildings geospatial data
HDB_PROPERTY <- RAW_HDB_PROPERTY %>%
mutate(RAW_HDB_PROPERTY, address = paste(blk_no, street))Obtain postal codes from OneMap API
addr_lst <- sort(unique(HDB_PROPERTY$address))
postal <- get_postal(addr_lst)
write_rds(postal, "data/fyp_preprocessing/hdb_property_postal.rds")Combine HDB Property data to HDB Buildings dataset
postal <- read_rds("data/fyp_preprocessing/hdb_property_postal.rds")
temp_hdb_property <- left_join(HDB_PROPERTY, postal, by=c("address" = "address"))temp_hdb_property <- mutate(temp_hdb_property, postal_check =
is.na(as.numeric(postal)) == FALSE)
temp_hdb_property %>% filter(postal_check == FALSE) blk_no street max_floor_lvl year_completed residential commercial
1 215 CHOA CHU KANG CTRL 11 1989 Y Y
2 216 CHOA CHU KANG CTRL 4 1989 Y N
market_hawker miscellaneous multistorey_carpark precinct_pavilion
1 N N N N
2 N N N N
bldg_contract_town total_dwelling_units X1room_sold X2room_sold X3room_sold
1 CCK 90 0 0 0
2 CCK 16 0 0 0
X4room_sold X5room_sold exec_sold multigen_sold studio_apartment_sold
1 90 0 0 0 0
2 16 0 0 0 0
X1room_rental X2room_rental X3room_rental other_room_rental
1 0 0 0 0
2 0 0 0 0
address postal postal_check
1 215 CHOA CHU KANG CTRL NIL FALSE
2 216 CHOA CHU KANG CTRL NIL FALSEtemp_hdb_property[temp_hdb_property$address == "215 CHOA CHU KANG CTRL", "postal"] = 680215
temp_hdb_property[temp_hdb_property$address == "216 CHOA CHU KANG CTRL", "postal"] = 680216
temp_hdb_property %>% filter(postal_check == FALSE) blk_no street max_floor_lvl year_completed residential commercial
1 215 CHOA CHU KANG CTRL 11 1989 Y Y
2 216 CHOA CHU KANG CTRL 4 1989 Y N
market_hawker miscellaneous multistorey_carpark precinct_pavilion
1 N N N N
2 N N N N
bldg_contract_town total_dwelling_units X1room_sold X2room_sold X3room_sold
1 CCK 90 0 0 0
2 CCK 16 0 0 0
X4room_sold X5room_sold exec_sold multigen_sold studio_apartment_sold
1 90 0 0 0 0
2 16 0 0 0 0
X1room_rental X2room_rental X3room_rental other_room_rental
1 0 0 0 0
2 0 0 0 0
address postal postal_check
1 215 CHOA CHU KANG CTRL 680215 FALSE
2 216 CHOA CHU KANG CTRL 680216 FALSE5 Combination of Data
5.1 OSM
osm_property_bldgs <- left_join(temp_osm, temp_hdb_property, by=c("postal" = "postal"))
write_rds(osm_property_bldgs, "data/fyp_preprocessing/osm_property_bldgs.rds")osm_property_bldgs <- read_rds("data/fyp_preprocessing/osm_property_bldgs.rds")tmap_options(check.and.fix = TRUE)
tmap_mode("view")
tm_shape(osm_property_bldgs %>% filter(bldg_contract_town == "CCK")) +
tm_fill(col = "cyan")